﻿using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Text;
using Spider_Demo.Models;
using DotnetSpider.Core;
using HtmlAgilityPack;
using DotnetSpider.Core.Scheduler;
using Spider_Demo.PageProcesser;
using Spider_Demo.Pipeline;

namespace Spider_Demo
{
    class Program
    {
        static void Main(string[] args)
        {
            Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
            //爬虫一
            //Spider_HtmlAgilityPack();

            //方案二
            Spider_DotNetSpider();
        }

        /// <summary>
        /// 使用“DotNetSpider”爬取图片数据
        /// </summary>
        private static void Spider_DotNetSpider()
        {
            //请求信息
            var site = new Site
            {
                CycleRetryTimes = 3,
                SleepTime = 200,
                //DownloadFiles = true,
                Headers = new Dictionary<string, string>()
                {
                    { "Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8" },
                    { "Cache-Control","no-cache" },
                    { "Connection","keep-alive" },
                    { "Content-Type","application/x-www-form-urlencoded; charset=UTF-8" },
                    { "User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.167 Safari/537.36"}
                }
            };

            #region 彼岸桌面的页面地址集合
            List<Request> bian_resList = new List<Request>();
            string bian_BaseUrl = "http://pic.netbian.com";
            HtmlWeb bian_htmlWeb = new HtmlWeb
            {
                OverrideEncoding = Encoding.GetEncoding("gb2312")
            };
            int bian_PageEnd = int.Parse(bian_htmlWeb.Load(bian_BaseUrl).DocumentNode.SelectSingleNode("//div[@class='page']/a[last()-1]").InnerText);
            for (int i = bian_PageEnd; i >= 1; i--)
            {
                Request res = new Request();
                if (i > 1)
                {
                    res.Url = string.Format(bian_BaseUrl + "/index_{0}.html", i);
                }
                else
                {
                    res.Url = $"{bian_BaseUrl}/index.html";
                }
                res.Method = HttpMethod.Get;
                bian_resList.Add(res);
            }
            #endregion

            #region MM131的页面地址
            List<Request> mm131_reslist = new List<Request>();
            string mm131_BaseUrl = "https://www.mm131.net";
            string[] MenuLists = { "/xinggan/list_6_{0}.html", "/qingchun/list_1_{0}.html", "/xiaohua/list_2_{0}.html", "/chemo/list_3_{0}.html", "/qipao/list_4_{0}.html", "/mingxing/list_5_{0}.html" };
            foreach (var item in MenuLists)
            {
                string BaseUrl = mm131_BaseUrl + item.Substring(0, item.LastIndexOf("/") + 1);
                HtmlWeb mm_htmlWeb = new HtmlWeb
                {
                    OverrideEncoding = Encoding.GetEncoding("gb2312")
                };
                var docHtml = mm_htmlWeb.Load(BaseUrl).DocumentNode.SelectSingleNode("//dl/dd[@class='page']/a[last()]").Attributes["href"].Value;
                //末页
                int PageEnd = int.Parse(docHtml.Substring(docHtml.LastIndexOf("_") + 1, docHtml.LastIndexOf(".") - docHtml.LastIndexOf("_") - 1));
                for (int i = PageEnd; i > 0; i--)
                {
                    Request res = new Request();
                    if (i > 1)
                    {
                        res.Url = string.Format($"{mm131_BaseUrl}{item}", i);
                    }
                    else
                    {
                        res.Url = BaseUrl;
                    }
                    res.Method = HttpMethod.Get;
                    mm131_reslist.Add(res);
                }
            }
            #endregion

            #region 单项任务
            //var spider = Spider.Create(site, new QueueDuplicateRemovedScheduler(), new netbian_Processor())
            //    .AddStartRequests(resList.ToArray())
            //    .AddPipeline(new Netbian_Pipeline());
            //spider.Downloader = new HttpClientDownloader();
            //spider.ThreadNum = 1;
            //spider.Run();
            #endregion

            #region 多项目任务
            List<Spider> spiders = new List<Spider>()
            {
               Spider.Create(site, new QueueDuplicateRemovedScheduler(), new NetBiAn_Processor()).AddStartRequests(bian_resList.ToArray()).AddPipeline(new NetBiAn_Pipeline()),
               Spider.Create(site,new QueueDuplicateRemovedScheduler(),new MM131_Processor()).AddStartRequests(mm131_reslist.ToArray()).AddPipeline(new MM131_Pipeline())
            };
            foreach (var item in spiders)
            {
                //item.Downloader = new HttpClientDownloader();// 爬虫下载器
                item.ThreadNum = 10;
                item.RunAsync();
            }
            #endregion

        }

        /// <summary>
        /// 使用“HtmlAgilityPack”爬取图片数据
        /// </summary>
        public static void Spider_HtmlAgilityPack()
        {
            string url = @"http://pic.netbian.com/4kfengjing/";
            HtmlWeb htmlWeb = new HtmlWeb
            {
                OverrideEncoding = Encoding.GetEncoding("gb2312")//编码
            };
            var htmlDoc = htmlWeb.Load(url);
            var nodes = htmlDoc.DocumentNode.SelectNodes("//*[@id='main']/div[3]/ul/li/a");
            List<BiAn_Images> images = new List<BiAn_Images>();
            foreach (var item in nodes)
            {
                string BaseAddUrl = "http://pic.netbian.com";
                var img = new BiAn_Images
                {
                    ImgTitle = item.SelectSingleNode("./b").InnerText,
                    Thumbnail_ImgUrl = BaseAddUrl + item.SelectSingleNode("./img").Attributes["src"].Value
                };
                HtmlWeb htmlWeb1 = new HtmlWeb();
                var info = htmlWeb1.Load($"http://pic.netbian.com{item.Attributes["href"].Value}");
                img.HD_ImgUrl = BaseAddUrl + info.DocumentNode.SelectSingleNode("//*[@id='img']/img").Attributes["src"].Value;

                images.Add(img);
                Console.WriteLine($"图片标题：{img.ImgTitle}，缩略图：{img.Thumbnail_ImgUrl}，下载地址：{img.HD_ImgUrl}");
            }
        }
    }
}
